package com.zju.cst.spider.utils;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class URLUtils {

    public static String getContent(String url){
        HttpURLConnection urlConn = null;
        InputStream in = null;
        BufferedReader bufin = null;
        StringBuilder sb = new StringBuilder();
        String str = null;
        try{
            urlConn = (HttpURLConnection) new URL(url).openConnection();

            urlConn.addRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
            urlConn.addRequestProperty("Accept-Language", "zh-CN,zh;q=0.8");
            urlConn.addRequestProperty("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:17.0) Gecko/20131029 Firefox/17.0");
            urlConn.addRequestProperty("DNT", "1");
            urlConn.addRequestProperty("Connection", "keep-alive");
            urlConn.addRequestProperty("Cache-Control", "max-age=0");

            in = urlConn.getInputStream();
            bufin = new BufferedReader(new InputStreamReader(in, "GBK"));
            while((str = bufin.readLine()) != null){
                sb.append(str+"\r\n");
            }
        }
        catch(Exception e){
            throw new RuntimeException(e);
        }finally{
            StreamUtils.close(bufin);
            StreamUtils.close(in);
        }
        return sb.toString();
    }
    
    public static List<String> regex(String content, String regex){
        List<String> list = new ArrayList<String>();
        Pattern pa = Pattern.compile(regex);
        Matcher ma=pa.matcher(content);
        while(ma.find()){
            list.add(ma.group());
        }
        return list;
    }
}
